subs = pysrt.open("Interstellar.srt")


DF = pd.DataFrame([
  {
    "Text": sub.text
} for sub in subs])

DF.head()

text = str(subs)
len(nltk.word_tokenize(text))

print(DF.shape)

DF['clean2'] = DF['Text'].apply(remove_noise)

DF = DF[DF['clean2'] != ""]

DF.head(10)

DF['score'] = DF['clean2'].apply(impurity)
DF.sort_values(by = ['score'], ascending = False)

DF['clean'] = DF['clean2'].apply(normalize)
DF.head(10)

# find all the unique tokens
# set is find unique
# nltk.word_tokenize is break down into words
# " ".join is combine into one long text
# .to_list() is a function to convert to list 
clean_tokens = set(nltk.word_tokenize(" ".join(DF['clean'].to_list())))

# what is wrong? 
misspelled = spell.unknown(clean_tokens)
print(misspelled)

output = []

# only the tagger and lemmatizer
for doc in nlp.pipe(DF['clean'].tolist(), disable=["tok2vec", "ner", "parser"]):
    tokens = textacy.extract.words(doc,
            filter_stops = True,           # default True, no stopwords
            filter_punct = True,           # default True, no punctuation
            filter_nums = True,            # default False, no numbers
            include_pos = None,            # default None = include all
            exclude_pos = None,            # default None = exclude none
            min_freq = 1)                  # minimum frequency of words
    # tokens_len = len([str(word) for word in tokens])
    # if tokens_len>0:
    output.append([str(word) for word in tokens]) # close output append 


output = [elem for elem in output if elem]
output[10]

# all items
print(type(output))
# first list
print(type(output[100]))
# first list, first item (this is the issue!)
print(type(output[100][0]))

word_counts = Counter(chain.from_iterable(output))

sorted_word_counts = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)

# Convert the sorted list back to a Counter object for presentation
sorted_word_counts_dict = Counter(dict(sorted_word_counts))

print(sorted_word_counts_dict)

text = str(subs)
DF = pd.DataFrame(
    # nltk.sent_tokenize(text),
    nltk.sent_tokenize(subtitle),
    columns = ["sentences"]
)

DF.head(20)

DF_POS.head(20)

DF_POS['upos'].value_counts()
# punct is not pos. so most common Nouns and verbs
#  x are thins it could not figure it out

DF_POS2 = pd.crosstab(DF_POS['token'], DF_POS['upos'])

# convert to true false to add up how many times not zero 
DF_POS2['total'] = DF_POS2.astype(bool).sum(axis=1)
#print out the rows that aren't 1 
DF_POS2[DF_POS2['total'] > 1]

# What are the 10 most common verb and nouns
nouns_df = DF_POS[DF_POS['upos']=='NOUN']
distinct_noun_count = nouns_df['token'].nunique()
top_10_nouns = nouns_df['token'].value_counts().head(10)
print('10 most common nouns are:\n', top_10_nouns)


verbs_df = DF_POS[DF_POS['upos']=='VERB']
distinct_verbs_count = verbs_df['token'].nunique()
top_10_verbs = verbs_df['token'].value_counts().head(10)
print('10 most common verbs are:\n', top_10_verbs)

# textacy KPE
# build an english language for textacy pipe
en = textacy.load_spacy_lang("en_core_web_sm", disable=("parser"))

# build a processor for textacy using spacy and process text
doc = textacy.make_spacy_doc(subtitle, lang = en)

# text rank algorithm  
print([kps for kps, weights in textacy.extract.keyterms.textrank(doc, normalize = "lemma",  topn = 5)])

terms = set([term for term, weight in textacy.extract.keyterms.textrank(doc)])
print(textacy.extract.utils.aggregate_term_variants(terms))

TR_keywords = keywords.keywords(subtitle, scores = True)
print(TR_keywords[0:20])

print(DF_NER['entity'].value_counts())

DF_NER2 = pd.crosstab(DF_NER['token'], DF_NER['entity'])
print(DF_NER2)

# convert to true false to add up how many times not zero 
DF_NER2['total'] = DF_NER2.astype(bool).sum(axis=1)
#print out the rows that aren't 1 
DF_NER2[DF_NER2['total'] > 1]

DF_dev

L_dev

DF_combined = pd.concat([DF_dev, pd.DataFrame(L_dev, columns = ["location1", "location2", "family1", "family2"])], axis = 1)
DF_combined.head()

DF_combined['location_yes'] = DF_combined['location1'] + DF_combined["location2"]
DF_combined['family_yes'] = DF_combined['family1'] + DF_combined["family2"]

print(DF_combined['location_yes'].value_counts())
print(DF_combined['family_yes'].value_counts())

# locations only
DF_loc = DF_combined[DF_combined['location_yes'] > 0]
DF_loc = DF_loc[['person1', 'person2']].reset_index(drop = True)

cooc_loc = DF_loc.groupby(by=["person1", "person2"], as_index=False).size()

# family only
DF_fam = DF_combined[DF_combined['family_yes'] > 0]
DF_fam = DF_fam[['person1', 'person2']].reset_index(drop = True)

cooc_fam = DF_fam.groupby(by=["person1", "person2"], as_index=False).size()

# take out issues where entity 1 == entity 2
cooc_loc = cooc_loc[cooc_loc['person1'] != cooc_loc['person2']]
cooc_fam = cooc_fam[cooc_fam['person1'] != cooc_fam['person2']]

print(cooc_loc.head())
print(cooc_fam.head())

# start by plotting the whole thing for location 
cooc_loc_small = cooc_loc[cooc_loc['size']>1]
graph = nx.from_pandas_edgelist(
           cooc_loc_small[['person1', 'person2', 'size']] \
           .rename(columns={'size': 'weight'}),
           source='person1', target='person2', edge_attr=True)

pos = nx.kamada_kawai_layout(graph, weight='weight')

_ = plt.figure(figsize=(20, 20))
nx.draw(graph, pos, 
        node_size=1000, 
        node_color='skyblue',
        alpha=0.8,
        with_labels = True)
plt.title('Graph Visualization', size=15)

for (node1,node2,data) in graph.edges(data=True):
    width = data['weight'] 
    _ = nx.draw_networkx_edges(graph,pos,
                               edgelist=[(node1, node2)],
                               width=width,
                               edge_color='#505050',
                               alpha=0.5)

plt.show()
plt.close()

cooc_fam
graph = nx.from_pandas_edgelist(
           cooc_fam[['person1', 'person2', 'size']] \
           .rename(columns={'size': 'weight'}),
           source='person1', target='person2', edge_attr=True)

pos = nx.kamada_kawai_layout(graph, weight='weight')

_ = plt.figure(figsize=(20, 20))
nx.draw(graph, pos, 
        node_size=1000, 
        node_color='skyblue',
        alpha=0.8,
        with_labels = True)
plt.title('Graph Visualization', size=15)

for (node1,node2,data) in graph.edges(data=True):
    width = data['weight'] 
    _ = nx.draw_networkx_edges(graph,pos,
                               edgelist=[(node1, node2)],
                               width=width,
                               edge_color='#505050',
                               alpha=0.5)

plt.show()
plt.close()